# -*- coding: utf-8 -*-
'''
Created on 28-11-2012

@author: LONG HOANG GIANG
'''

from CrawlerLib import commonlib
from Model.storymodelv2 import StoryModel
import os
import re
import json
import workerpool


def getDetail(url):
    tree = commonlib.loadweb(url).build_tree()
    contentNode = tree.xpath("//div[@class='entry-content clearfix']")
    html = commonlib.html2text(contentNode[0]).strip().encode('utf-8')
    html = re.sub(r"\n", "<br />", html)
    return html
    
def process():
    
    db = StoryModel(DATABASE)
    db.open(True)
    
    fp = open('/longhoanggiang/cache/tongtailslink', 'r')
    data = json.load(fp)
    fp.close()
    for item in data:
        detail = getDetail(item['url'])
        db.add_story(commonlib.toUpper(item['title']), detail, 0)
        print '____________________________________________'
        print item['title']
        print '____________________________________________'
        print detail, "\n\n"
    
    db.close()

def getLinks():
    
    tree = commonlib.loadweb(URL).build_tree()
    try:
        fp = open('/longhoanggiang/cache/tongtailslink', 'r')
        data = json.load(fp)
        fp.close()
    except:
        data = None
    if data == None:
        data = []
        for item in tree.xpath("//ul[@class='lcp_catlist']/li/a"):
            title = re.sub(ur"(Nợ em một hạnh phúc *)|[\[\]]", '', item.text).strip(':').strip()
            title = title.replace('Nợ em một hạnh phúc ', '').strip()
            title = re.sub(r": *", ": ", title)
            title = re.sub(ur"Ch.(\d+)", "Chương \\1", title)
            title = title.replace(' - ', '-').strip('-').strip()
            if 'Bình luận của độc giả' in title or 'Độc giả bình luận' in title or 'thông báo' in title or 'Thông báo' in title: continue
            link = item.get('href', '')
            if link == '': continue
            data.append({'title': title, 'url': link})
        data.reverse()
        fp = open('/longhoanggiang/cache/tongtailslink', 'w')
        json.dump(data, fp)
        fp.close()
    
    def load(item):
        commonlib.loadweb(item['url'])
    pool = workerpool.WorkerPool(size=10)
    pool.map(load, data)
    pool.shutdown()
    pool.wait()
    
    return data
        
    

if __name__ == '__main__':
    
    URL = 'http://vficland.com/truyen-phuong-dong/no-em-mot-hanh-phuc/'
    DATABASE = '/longhoanggiang/database/{DB}'.format(DB='noem1hanhphuc')
#    getDetail('http://vficland.com/tong-tai-thuc-dang-so-ngoai-truyen-cuoi-tieu-vuong-tu-ket-thuc-mo-hoan/')
    getLinks()

    process()
    
    
    os._exit(1)